import requests
import re
from lxml import etree
import pymysql
import threading
import time
import A地区
from concurrent.futures import ThreadPoolExecutor,as_completed,Future
import Sha1Util

# 关联表area，获取到 id,cityHref
sql = "select id,cityHref from area"
conn = pymysql.connect(host='localhost', user='root', password='0830', database='moonlightcat_test')
cursor = conn.cursor()

lock = threading.RLock()  # 获取锁


# 定义获取area表的数据的方法（id，cityHref）
def getIDAndcityHrefFromCity():
    cursor.execute(sql)
    result = cursor.fetchall()
    return result


def jianzhi(href_tuple):
    cityId = href_tuple[2]
    headers = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36 Edg/95.0.1020.40"
    }
    resp = requests.get(url=href_tuple[1], headers=headers)
    resp.encoding = "utf-8"
    xian_text = resp.text
    tree = etree.HTML(xian_text)
    xian_name = tree.xpath('//div[@class="content_list_box"]//li/a/@title')
    xian_href = tree.xpath('//div[@class="content_list_box"]//li/a/@href')
    xian_people = tree.xpath('//ul[@class="content_list_wrap"]//li/div[@class="left visited"]/span/@title')
    xian_time = tree.xpath('//ul[@class="content_list_wrap"]//li/div[@class="left date"]/@title')

    city_href = href_tuple[1]
    city_href1 = "https://" + str(city_href).split("/")[-3]
    if len(xian_name) == 1:
        xian_name.append("")
    for i in range(len(xian_name)):
        xian_name1 = xian_name[i]
        xian_href1 = city_href1 + xian_href[i]
        id = Sha1Util.jia_mi(xian_name1 + xian_href1)
        xian_people1 = xian_people[i]
        xian_time1 = xian_time[i]
        if "刚" in xian_time1:
            xian_time1 = time.strftime('%Y-%m-%d', time.localtime())
        if "天" in xian_time1:
            xian_time1 = time.strftime('%Y-%m-%d', time.localtime())
        if "前" in xian_time1:
            xian_time1 = time.strftime('%Y-%m-%d', time.localtime())
        insert_sql = "insert ignore into jian(id,name,href,cityId,people,time)values('%s','%s','%s','%s','%s','%s')" % (
            id, xian_name1, xian_href1, cityId, xian_people1, xian_time1)
        # 访问网站不是200时直接跳出
        stu = resp.status_code
        if stu != 200:
            break
        try:
            lock.acquire()  # 加锁
            cursor.execute(insert_sql)
            conn.commit()
            print("插入数据成功！", xian_name1, xian_href1)
            lock.release()  # 解锁
        except Exception as e:
            print(f"数据插入失败！e:{e},sql:{sql}")
            conn.rollback()


time.sleep(0.05)


def duoxiancheng():
    result = A地区.getDatafromxian()  # id,cityHref
    th = ThreadPoolExecutor(14)
    all_task  =list()
    for data in result:
        # 拼接网址 把area表的县网址进行切割
        city_href1 = str(data[1]).split("/")[-2]
        # 切割网址，获取到城市字段
        xian_href2 = str(data[1]).split("/")[2]
        for i in range(1, 11):
            page_href = f"https://%s/%s/index{i}.html" % (xian_href2, city_href1)
            print(page_href)
            args = (page_href, data[1],data[0])
            all_task.append(th.submit(jianzhi,args))
            time.sleep(0.05)

    for future in as_completed(all_task):
        data = future.result()
        print("in main: get page {}s success".format(data))

    print("结束")

if __name__ == '__main__':
    duoxiancheng()
